package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/shirou/gopsutil/cpu"
	"github.com/shirou/gopsutil/disk"
	"github.com/shirou/gopsutil/mem"
	"github.com/shirou/gopsutil/net"
	"github.com/shirou/gopsutil/process"
	"io/ioutil"
	"math"
	//tcpnet "net"
	"os"
	"os/exec"
	pathpkg "path"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"
)

const FT int64 = 1000000

func IsNull(t *Status) bool {
	return t == nil || t.ProcCount == 0
}

/*func checkNet(nc *NetCheck, ip, port, clu, proj, tag string) {
	nc.lastScanTime = time.Now().UnixNano() / FT
	p, _ := strconv.Atoi(port)
	tcpAddr := tcpnet.TCPAddr{
		IP:   tcpnet.ParseIP(ip),
		Port: p,
	}

	conn, err := tcpnet.DialTCP("tcp", nil, &tcpAddr)

	if err == nil {
		nc.metricStr = append(nc.metricStr, fmt.Sprintf(`net_check{%sdest_host="%s",port="%s",subcluster="%s",subproj="%s",m="isConnected"} 1 %d`, tag, ip, port, clu, proj, nc.lastScanTime))
		conn.Close()
	} else {
		nc.metricStr = append(nc.metricStr, fmt.Sprintf(`net_check{%sdest_host="%s",port="%s",subcluster="%s",subproj="%s",m="isConnected"} -1 %d`, tag, ip, port, clu, proj, nc.lastScanTime))
	}
	nc.wgp.Done()
}*/

func checkNet(nc *NetCheck, ip, port, clu, proj, tag string) {
	nc.lastScanTime = time.Now().UnixNano() / FT
	cmdstr := `echo -e "\n"|telnet ` + ip + ` ` + port + `|grep Connected`
	cmd := exec.Command(`/bin/sh`, `-c`, cmdstr)
	out, _ := cmd.CombinedOutput()
	cmdout := string(out)
	if strings.Contains(cmdout, `refused`) {
		nc.metricStr = append(nc.metricStr, fmt.Sprintf(`net_check{%sdest_host="%s",port="%s",subcluster="%s",subproj="%s",m="isConnected"} -1 %d`, tag, ip, port, clu, proj, nc.lastScanTime))
	}
	if strings.Contains(cmdout, `Connected`) {
		nc.metricStr = append(nc.metricStr, fmt.Sprintf(`net_check{%sdest_host="%s",port="%s",subcluster="%s",subproj="%s",m="isConnected"} 1 %d`, tag, ip, port, clu, proj, nc.lastScanTime))
	}
	nc.wgp.Done()
}

func (nc NetCheck) getMetric(tag string) []string {
	if nc.cks == nil {
		log.Error("no net_check!")
		return nil
	}
	nc.metricStr = make([]string, 0)
	nc.metricStr = append(nc.metricStr, "# TYPE net_check gauge")
	for i, _ := range nc.cks.MustArray() {
		cf := nc.cks.GetIndex(i)
		cp := cf.Get("cluster_proj").MustString()
		c_p := strings.Split(cp, `#`)
		clu := c_p[0]
		proj := c_p[1]
		ipt := cf.Get("ip_port").MustStringArray()
		for _, v := range ipt {
			ss := strings.Split(v, `:`)
			ip := ss[0]
			port := ss[1]
			nc.wgp.Add(1)
			go checkNet(&nc, ip, port, clu, proj, tag)
		}
	}
	nc.wgp.Wait()
	return nc.metricStr
}

func pingIP(pg *Ping, ip, tag string) {
	pg.lastScanTime = time.Now().UnixNano() / FT
	cmdstr := `ping ` + ip + ` -c 1 -W 1|grep rtt|awk '{print $4}'`
	var out bytes.Buffer
	cmd := exec.Command(`/bin/sh`, `-c`, cmdstr)
	cmd.Stdout = &out
	err := cmd.Run()
	checkErr(err, "ping "+ip+" error!")
	sent := out.String()
	if sent == "" {
		pg.metricStr = append(pg.metricStr, fmt.Sprintf(`net_ping{%sdest_host="%s",m="down"} -1 %d`, tag, ip, pg.lastScanTime))
		pg.wg.Done()
		return
	}
	pv := strings.Split(strings.TrimSpace(sent), `/`)
	pt, _ := strconv.ParseFloat(pv[0], 64)
	pg.metricStr = append(pg.metricStr, fmt.Sprintf(`net_ping{%sdest_host="%s",m="ping_time",u="ms"} %.2f %d`, tag, ip, pt, pg.lastScanTime))
	pg.wg.Done()
}

func (pg Ping) getMetric(tag string) []string {
	if pg.ips == "" {
		log.Error("no ping ips!")
		return nil
	}
	pg.metricStr = make([]string, 0)
	pg.metricStr = append(pg.metricStr, "# TYPE net_ping gauge")
	ips := strings.Split(pg.ips, `/`)
	for _, ip := range ips {
		pg.wg.Add(1)
		go pingIP(&pg, ip, tag)
	}
	pg.wg.Wait()
	return pg.metricStr
}

func (host *Host) refresh(interval int64) {
	_, ok := host.tags["host"]
	if !ok {
		cmd := exec.Command("/bin/hostname", "-f")
		var out bytes.Buffer
		cmd.Stdout = &out
		err := cmd.Run()
		if err != nil {
			log.Error(`run "hostname -f":`, err)
			host.tags["host"], _ = os.Hostname()
		} else {
			fqdn := out.String()
			fqdn = fqdn[:len(fqdn)-1]
			host.tags["host"] = fqdn
		}
	}
	var buf bytes.Buffer
	for k, v := range host.tags {
		buf.WriteString(fmt.Sprintf(`%s="%s",`, k, v))
	}
	str := buf.String()
	tick := time.Tick(time.Millisecond * time.Duration(interval))
	for {
		select {
		case <-tick:
			host.mdata = make([]string, 0)
			for _, m := range host.metrics {
				go host.loadMetric(m.getMetric(str))
			}
		case <-host.stopping:
			host.mu.Lock()
			host.mu.Unlock()
			close(host.stopped)
		}
	}
}

func (proc *Proc) refresh(interval int64) {
	_, ok := proc.tags["host"]
	if !ok {
		cmd := exec.Command("/bin/hostname", "-f")
		var out bytes.Buffer
		cmd.Stdout = &out
		err := cmd.Run()
		if err != nil {
			log.Error(`run "hostname -f":`, err)
			proc.tags["host"], _ = os.Hostname()
		} else {
			fqdn := out.String()
			fqdn = fqdn[:len(fqdn)-1]
			proc.tags["host"] = fqdn
		}
	}
	var buf bytes.Buffer
	for k, v := range proc.tags {
		buf.WriteString(fmt.Sprintf(`%s="%s",`, k, v))
	}
	str := buf.String()
	tick := time.Tick(time.Millisecond * time.Duration(interval))
	for {
		select {
		case <-tick:
			proc.mdata = make([]string, 0)
			for _, m := range proc.metrics {
				go proc.loadMetric(m.getMetric(str))
			}
		case <-proc.stopping:
			proc.mu.Lock()
			proc.mu.Unlock()
			close(proc.stopped)
		}
	}
}

func (host *Host) Close() {
	close(host.stopping)
	<-host.stopped
}

func (proc *Proc) Close() {
	close(proc.stopping)
	<-proc.stopped
}

func (host *Host) loadMetric(metricStr []string) {
	host.mu.Lock()
	defer host.mu.Unlock()
	for _, m := range metricStr {
		host.mdata = append(host.mdata, m)
	}
}

func (proc *Proc) loadMetric(metricStr []string) {
	proc.mu.Lock()
	defer proc.mu.Unlock()
	for _, m := range metricStr {
		proc.mdata = append(proc.mdata, m)
	}
}

func (host *Host) export() string {
	var buf bytes.Buffer
	host.mu.RLock()
	defer host.mu.RUnlock()
	for _, v := range host.mdata {
		buf.WriteString(v)
		buf.WriteString("\n")
	}
	return buf.String()
}

func (proc *Proc) export() string {
	var buf bytes.Buffer
	proc.mu.RLock()
	defer proc.mu.RUnlock()
	for _, v := range proc.mdata {
		buf.WriteString(v)
		buf.WriteString("\n")
	}
	return buf.String()
}

var cpuOld = make(map[string]float64)
var netOld = make(map[string]uint64)
var diskOld = make(map[string]uint64)

func (hb HostBase) getMetric(tag string) []string {
	hb.lastScanTime = time.Now().UnixNano() / FT
	hb.metricStr = make([]string, 0)
	hb.metricStr = append(hb.metricStr, "# TYPE cpu_info gauge")
	//cpu
	info, err := cpu.Info()
	checkErr(err, "cpu.Info() error!")
	var cores int32 = 0
	for _, c := range info {
		cores = cores + c.Cores
	}
	hb.metricStr = append(hb.metricStr, fmt.Sprintf(`cpu_info{%sm="cores",u="cores"} %d %d`, tag, cores, hb.lastScanTime))

	v, err := cpu.Times(false)
	if err != nil || len(v) != 1 {
		log.Error("cpu.Times() error!")
	}
	cpuNew := make(map[string]float64)
	cpuNew["idle"] = v[0].Idle
	cpuNew["total"] = v[0].User + v[0].System + v[0].Nice + v[0].Iowait + v[0].Irq + v[0].Softirq + v[0].Steal + v[0].Guest + v[0].GuestNice + v[0].Stolen + v[0].Idle
	ret := PercentMinus(cpuOld, cpuNew)
	cpuOld = cpuNew
	usedcores := float64(cores) * ret["used"]
	c := math.Floor(usedcores)
	if usedcores-c > 0.1 {
		usedcores = c + 1
	} else {
		usedcores = c
	}
	hb.metricStr = append(hb.metricStr, fmt.Sprintf(`cpu_info{%sm="used_cores",u="cores"} %d %d`, tag, int64(usedcores), hb.lastScanTime))
	hb.metricStr = append(hb.metricStr, "# TYPE cpu gauge")
	for k, v := range ret {
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`cpu{%sm="%s",u="persent"} %.4f %d`, tag, k, v, hb.lastScanTime))
	}
	//mem
	vm, err := mem.VirtualMemory()
	checkErr(err, "mem.VirtualMemory() error!")
	hb.metricStr = append(hb.metricStr, "# TYPE mem_info gauge")
	if vm.Total != 0 {
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`mem_info{%sm="total",u="GB"} %.2f %d`, tag, float64(vm.Total)/1024/1024/1024, hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`mem_info{%sm="used",u="GB"} %.2f %d`, tag, float64(vm.Used)/1024/1024/1024, hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, "# TYPE mem gauge")
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`mem{%sm="used",u="persent"} %.4f %d`, tag, float64(vm.Used)/float64(vm.Total), hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`mem{%sm="available",u="persent"} %.4f %d`, tag, float64(vm.Available)/float64(vm.Total), hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`mem{%sm="buffers",u="persent"} %.4f %d`, tag, float64(vm.Buffers)/float64(vm.Total), hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`mem{%sm="cached",u="persent"} %.4f %d`, tag, float64(vm.Cached)/float64(vm.Total), hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`mem{%sm="shared",u="persent"} %.4f %d`, tag, float64(vm.Shared)/float64(vm.Total), hb.lastScanTime))
	}
	sm, err := mem.SwapMemory()
	checkErr(err, "mem.SwapMemory() error!")
	if sm.Total != 0 {
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`mem{%sm="swap_used",u="persent"} %.4f %d`, tag, float64(sm.Used)/float64(sm.Total), hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`mem{%sm="swap_free",u="persent"} %.4f %d`, tag, float64(sm.Free)/float64(sm.Total), hb.lastScanTime))
	}
	interval := float64(hb.interval) / 1000
	//disk
	d, err := disk.Partitions(false)
	checkErr(err, "disk.Partitions() error!")
	hb.metricStr = append(hb.metricStr, "# TYPE disk_info gauge")
	for _, v := range d {
		u, err := disk.Usage(v.Mountpoint)
		if err != nil {
			continue
		}
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`disk_info{%spath="%s",m="space",u="GB"} %.2f %d`, tag, v.Device, float64(u.Total)/1024/1024/1024, hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`disk_info{%spath="%s",m="used",u="GB"} %.2f %d`, tag, v.Device, float64(u.Used)/1024/1024/1024, hb.lastScanTime))
	}
	hb.metricStr = append(hb.metricStr, "# TYPE disk_usage gauge")
	for _, v := range d {
		u, err := disk.Usage(v.Mountpoint)
		if err != nil {
			continue
		}
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`disk_usage{%spath="%s",m="space",u="persent"} %.4f %d`, tag, v.Mountpoint, u.UsedPercent/100, hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`disk_usage{%spath="%s",m="inode",u="persent"} %.4f %d`, tag, v.Mountpoint, u.InodesUsedPercent/100, hb.lastScanTime))
	}
	cmdstr := `du -s /var|awk '{print $1}'`
	cmd := exec.Command(`/bin/sh`, `-c`, cmdstr)
	out, _ := cmd.CombinedOutput()
	cmdout := string(out)
	sp, _ := strconv.ParseFloat(strings.TrimSpace(cmdout), 64)
	hb.metricStr = append(hb.metricStr, fmt.Sprintf(`disk_usage{%spath="/var",m="used",u="GB"} %.2f %d`, tag, sp/1024/1024, hb.lastScanTime))

	diskNowValues := make(map[string]uint64)
	dcs, err := disk.IOCounters()
	checkErr(err, "disk.IOCounters() error!")
	for name, dc := range dcs {
		diskNowValues[name+"rcount"] = dc.ReadCount
		diskNowValues[name+"wcount"] = dc.WriteCount
		diskNowValues[name+"mrcount"] = dc.MergedReadCount
		diskNowValues[name+"mwcount"] = dc.MergedWriteCount
		diskNowValues[name+"rbytes"] = dc.ReadBytes
		diskNowValues[name+"wbytes"] = dc.WriteBytes
		diskNowValues[name+"rtime"] = dc.ReadTime
		diskNowValues[name+"wtime"] = dc.WriteTime
		diskNowValues[name+"iotime"] = dc.IoTime
		diskNowValues[name+"weightedIO"] = dc.WeightedIO
	}
	diskValues := CounterMinus(diskOld, diskNowValues)
	diskOld = diskNowValues
	hb.metricStr = append(hb.metricStr, "# TYPE disk_iorate gauge")
	for name, _ := range dcs {
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`disk_iorate{%spath="%s",m="%s"} %.2f %d`, tag, name, "r/s", diskValues[name+"rcount"]/interval, hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`disk_iorate{%spath="%s",m="%s"} %.2f %d`, tag, name, "w/s", diskValues[name+"wcount"]/interval, hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`disk_iorate{%spath="%s",m="%s"} %.2f %d`, tag, name, "rkB/s", diskValues[name+"rbytes"]/1024/interval, hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`disk_iorate{%spath="%s",m="%s"} %.2f %d`, tag, name, "wkB/s", diskValues[name+"wbytes"]/1024/interval, hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`disk_iorate{%spath="%s",m="%s"} %.2f %d`, tag, name, "tps", (diskValues[name+"rcount"]+diskValues[name+"wcount"])/interval, hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`disk_iorate{%spath="%s",m="%s"} %.2f %d`, tag, name, "util", diskValues[name+"iotime"]/float64(hb.interval), hb.lastScanTime))
	}
	//net
	info2, err := net.Interfaces()
	checkErr(err, "net.Interfaces() error!")
	hb.metricStr = append(hb.metricStr, "# TYPE net_info gauge")
	hb.netInfo = make(map[string]float64)
	for _, v := range info2 {
		cmd1 := exec.Command(`ethtool`, v.Name)
		cmd2 := exec.Command(`grep`, `Speed`)
		var out bytes.Buffer
		cmd2.Stdin, _ = cmd1.StdoutPipe()
		cmd2.Stdout = &out
		cmd2.Start()
		cmd1.Run()
		cmd2.Wait()
		stat := out.String()
		if !strings.Contains(stat, `Speed:`) {
			continue
		}
		speed, err := parse(stat)
		if err != nil {
			continue
		}
		hb.netInfo[v.Name] = speed
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`net_info{%spath="%s",m="speed",u="Mb/s"} %.2f %d`, tag, v.Name, speed, hb.lastScanTime))
	}
	ncs, err := net.IOCounters(true)
	checkErr(err, "net.IOCounters() error!")
	netNowValues := make(map[string]uint64)
	for _, nc := range ncs {
		name := nc.Name
		netNowValues[name+"_bytes_sent"] = nc.BytesSent
		netNowValues[name+"_bytes_recv"] = nc.BytesRecv
		netNowValues[name+"_packets_sent"] = nc.PacketsSent
		netNowValues[name+"_packets_recv"] = nc.PacketsRecv
		netNowValues["all_bytes_sent"] += nc.BytesSent
		netNowValues["all_bytes_recv"] += nc.BytesRecv
		netNowValues["all_packets_sent"] += nc.PacketsSent
		netNowValues["all_packets_recv"] += nc.PacketsRecv
	}
	netValues := CounterMinus(netOld, netNowValues)
	netOld = netNowValues
	netret := make(map[string]float64)
	for _, nc := range ncs {
		netret[fmt.Sprintf(`path="%s",m="%s"`, nc.Name, "bytes_sent/s")] = netValues[nc.Name+"_bytes_sent"] / interval
		netret[fmt.Sprintf(`path="%s",m="%s"`, nc.Name, "bytes_recv/s")] = netValues[nc.Name+"_bytes_recv"] / interval
		netret[fmt.Sprintf(`path="%s",m="%s"`, nc.Name, "packets_sent/s")] = netValues[nc.Name+"_packets_sent"] / interval
		netret[fmt.Sprintf(`path="%s",m="%s"`, nc.Name, "packets_recv/s")] = netValues[nc.Name+"_packets_recv"] / interval
	}
	netret[fmt.Sprintf(`path="all",m="%s"`, "bytes_sent/s")] = netValues["all_bytes_sent"] / interval
	netret[fmt.Sprintf(`path="all",m="%s"`, "bytes_recv/s")] = netValues["all_bytes_recv"] / interval
	netret[fmt.Sprintf(`path="all",m="%s"`, "packets_sent/s")] = netValues["all_packets_sent"] / interval
	netret[fmt.Sprintf(`path="all",m="%s"`, "packets_recv/s")] = netValues["all_packets_recv"] / interval
	fs, err := ioutil.ReadDir("/sys/class/net")
	checkErr(err, "read /sys/class/net error!")
	for _, v := range fs {
		if _, ok := hb.netInfo[v.Name()]; !ok {
			continue
		}
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`net_info{%spath="%s",m="sentPersent"} %.4f %d`, tag, v.Name(), netret[fmt.Sprintf(`path="%s",m="bytes_sent/s"`, v.Name())]/131072/hb.netInfo[v.Name()], hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`net_info{%spath="%s",m="recvPersent"} %.4f %d`, tag, v.Name(), netret[fmt.Sprintf(`path="%s",m="bytes_recv/s"`, v.Name())]/131072/hb.netInfo[v.Name()], hb.lastScanTime))
	}
	hb.metricStr = append(hb.metricStr, "# TYPE net_iorate gauge")
	for _, nc := range ncs {
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`net_iorate{%spath="%s",m="bytes_sent/s"} %.2f %d`, tag, nc.Name, netValues[nc.Name+"_bytes_sent"]/interval, hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`net_iorate{%spath="%s",m="bytes_recv/s"} %.2f %d`, tag, nc.Name, netValues[nc.Name+"_bytes_recv"]/interval, hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`net_iorate{%spath="%s",m="packets_sent/s"} %.2f %d`, tag, nc.Name, netValues[nc.Name+"_packets_sent"]/interval, hb.lastScanTime))
		hb.metricStr = append(hb.metricStr, fmt.Sprintf(`net_iorate{%spath="%s",m="packets_recv/s"} %.2f %d`, tag, nc.Name, netValues[nc.Name+"_packets_recv"]/interval, hb.lastScanTime))
	}
	hb.metricStr = append(hb.metricStr, fmt.Sprintf(`net_iorate{%spath="all",m="bytes_sent/s"} %.2f %d`, tag, netValues["all_bytes_sent"]/interval, hb.lastScanTime))
	hb.metricStr = append(hb.metricStr, fmt.Sprintf(`net_iorate{%spath="all",m="bytes_recv/s"} %.2f %d`, tag, netValues["all_bytes_recv"]/interval, hb.lastScanTime))
	hb.metricStr = append(hb.metricStr, fmt.Sprintf(`net_iorate{%spath="all",m="packets_sent/s"} %.2f %d`, tag, netValues["all_packets_sent"]/interval, hb.lastScanTime))
	hb.metricStr = append(hb.metricStr, fmt.Sprintf(`net_iorate{%spath="all",m="packets_recv/s"} %.2f %d`, tag, netValues["all_packets_recv"]/interval, hb.lastScanTime))
	return hb.metricStr
}

func (pb ProcBase) getMetric(tag string) []string {
	procs := make(map[string]*ProcessItem)
	pb.lastScanTime = time.Now().UnixNano() / FT
	f, err := os.Stat(pb.confdir)
	checkErr(err, "pb.confdir stat error!")
	if !f.IsDir() {
		b, err := ioutil.ReadFile(pb.confdir)
		checkErr(err, "read proc config error!")
		err = json.Unmarshal(b, &procs)
		checkErr(err, "proc change to json error!")
	} else {
		files, err := ioutil.ReadDir(pb.confdir)
		checkErr(err, "read proc configdir error!")
		for _, f := range files {
			if f.Name() != "mas.json" {
				path := pathpkg.Join(pb.confdir, f.Name())
				b, err := ioutil.ReadFile(path)
				checkErr(err, "read proc configfile error!")
				ps := make(map[string]*ProcessItem)
				err = json.Unmarshal(b, &ps)
				checkErr(err, "proc change to json error!")
				for k, v := range ps {
					procs[k] = v
				}
			}
		}
	}
	files, err := ioutil.ReadDir("/proc")
	checkErr(err, "scan process error!")
	for _, p := range procs {
		p.Reset()
	}
	for _, f := range files {
		if !f.IsDir() {
			continue
		}
		re, rerr := regexp.Compile("[0-9]+")
		checkErr(rerr, "parse pid error!")
		if !re.MatchString(f.Name()) {
			continue
		}
		pid, err := strconv.Atoi(f.Name())
		if err != nil || pid <= 1 {
			continue
		}
		b, ctime, err := ParseCmdline(pid)
		if err != nil {
			continue
		}
		for _, p := range procs {
			cmdline := string(b)
			if p.reg == nil {
				p.reg, err = regexp.Compile(p.ProcessPattern)
				checkErr(err, "compile pattern error!")
			}
			if !p.reg.MatchString(cmdline) {
				continue
			}
			err := p.process.Add(ctime, pid)
			checkErr(err, "process add error!")
		}
	}
	value := make(map[string]*pwMetric)
	for pname, v := range procs {
		if _, ok := value[pname]; !ok {
			value[pname] = &pwMetric{}
		}
		if _, ye := v.Tags["port"]; !ye {
			v.Tags["port"] = ""
		}
		stat, err := v.process.Status(v.Tags["port"])
		checkErr(err, "process.Status() error!")
		if IsNull(stat) {
			value[pname].stat = nil
			value[pname].duration = 0
			continue
		}
		value[pname].stat = stat
		value[pname].duration = v.process.Uptime()
	}
	interval := float64(pb.interval) / 1000
	var buf bytes.Buffer
	ret := make(map[string][]string)
	for pname, v := range procs {
		for key, val := range v.Tags {
			if val == "" {
				continue
			}
			buf.WriteString(fmt.Sprintf(`%s="%s",`, key, val))
		}
		proc_msg := buf.String()
		name := fmt.Sprintf(`%s%sproc_name="%s"`, tag, proc_msg, pname)
		buf.Reset()
		//format := "%s{%s} %d %d"
		format1 := `%s{%s,m="%s",u="%s"} %.2f %d`
		format3 := `%s{%s,m="%s",u="%s"} %d %d`
		format2 := `%s{%s,m="%s"} %d %d`

		if IsNull(value[pname].stat) {
			ret["proc_stat"] = append(ret["proc_stat"],
				fmt.Sprintf(format2, "proc_stat", name, "isRunning", 0, pb.lastScanTime))
			continue
		}
		duration := value[pname].duration
		stat := value[pname].stat
		ret["proc_stat"] = append(ret["proc_stat"],
			fmt.Sprintf(format2, "proc_stat", name, "isRunning", 1, pb.lastScanTime))
		ret["proc_uptime"] = append(ret["proc_uptime"],
			fmt.Sprintf(`%s{%s,u="%s"} %d %d`, "proc_uptime", name, "ms", duration, pb.lastScanTime))

		ret["proc_cpu"] = append(ret["proc_cpu"],
			fmt.Sprintf(format3, "proc_cpu", name, "proc_cutime", "ms", stat.CuTime, pb.lastScanTime))

		ret["proc_cpu"] = append(ret["proc_cpu"],
			fmt.Sprintf(format3, "proc_cpu", name, "proc_cstime", "ms", stat.CsTime, pb.lastScanTime))

		ret["proc_cpu"] = append(ret["proc_cpu"],
			fmt.Sprintf(format3, "proc_cpu", name, "proc_stime", "ms", stat.STime, pb.lastScanTime))

		ret["proc_cpu"] = append(ret["proc_cpu"],
			fmt.Sprintf(format3, "proc_cpu", name, "proc_utime", "ms", stat.UTime, pb.lastScanTime))

		ret["proc_cpu"] = append(ret["proc_cpu"],
			fmt.Sprintf(`%s{%s,m="%s",u="%s"} %.2f %d`, "proc_cpu", name, "cpu_usage", "persent", stat.CpuUsage, pb.lastScanTime))

		ret["proc_memory"] = append(ret["memory"],
			fmt.Sprintf(format1, "proc_memory_size", name, "memory_size", "kB", float64(stat.MemSize)/1024, pb.lastScanTime))

		ret["proc_thread_count"] = append(ret["proc_thread_count"],
			fmt.Sprintf(format2, "proc_thread_count", name, "proc_thread_count", stat.ThreadCount, pb.lastScanTime))

		ret["proc_count"] = append(ret["proc_count"],
			fmt.Sprintf(format2, "proc_count", name, "proc_count", stat.ProcCount, pb.lastScanTime))

		ret["proc_diskIO"] = append(ret["proc_diskIO"],
			fmt.Sprintf(format1, "proc_diskIO", name, "read_bytes", "kB/s", float64(stat.ReadBytes)/1024/interval, pb.lastScanTime))

		ret["proc_diskIO"] = append(ret["proc_diskIO"],
			fmt.Sprintf(format1, "proc_diskIO", name, "write_bytes", "kB/s", float64(stat.WriteBytes)/1024/interval, pb.lastScanTime))

		ret["proc_diskIO"] = append(ret["proc_diskIO"],
			fmt.Sprintf(format1, "proc_diskIO", name, "read_count", "r/s", float64(stat.ReadCount)/interval, pb.lastScanTime))

		ret["proc_diskIO"] = append(ret["proc_diskIO"],
			fmt.Sprintf(format1, "proc_diskIO", name, "write_count", "w/s", float64(stat.WriteCount)/interval, pb.lastScanTime))

		ret["proc_netIO"] = append(ret["proc_netIO"],
			fmt.Sprintf(format1, "proc_netIO", name, "bytes_sent", "kB/s", float64(stat.BytesSent)/1024/interval, pb.lastScanTime))

		ret["proc_netIO"] = append(ret["proc_netIO"],
			fmt.Sprintf(format1, "proc_netIO", name, "bytes_recv", "kB/s", float64(stat.BytesRecv)/1024/interval, pb.lastScanTime))

		ret["proc_netIO"] = append(ret["proc_netIO"],
			fmt.Sprintf(format1, "proc_netIO", name, "packets_sent", "packet/s", float64(stat.PacketsSent)/interval, pb.lastScanTime))

		ret["proc_netIO"] = append(ret["proc_netIO"],
			fmt.Sprintf(format1, "proc_netIO", name, "packets_recv", "packet/s", float64(stat.PacketsRecv)/interval, pb.lastScanTime))

	}
	for k, v := range ret {
		s := fmt.Sprintf("# TYPE %s gauge", k)
		if k == "proc_uptime" {
			s = fmt.Sprintf("# TYPE %s counter", k)
		}
		pb.metricStr = append(pb.metricStr, s)
		for _, l := range v {
			pb.metricStr = append(pb.metricStr, l)
		}
	}
	return pb.metricStr
}

func ProcCounterMinus(last, now *Status) *Status {
	ret := &Status{}
	if last == nil {
		return ret
	}
	ret.ProcType = now.ProcType
	ret.ProcCount = now.ProcCount
	ret.ThreadCount = now.ThreadCount
	ret.CuTime = now.CuTime - last.CuTime
	ret.CsTime = now.CsTime - last.CsTime
	ret.STime = now.STime - last.STime
	ret.UTime = now.UTime - last.UTime
	ret.CheckTime = now.CheckTime - last.CheckTime
	ret.MemSize = now.MemSize
	ret.ReadCount = now.ReadCount - last.ReadCount
	ret.WriteCount = now.WriteCount - last.WriteCount
	ret.ReadBytes = now.ReadBytes - last.ReadBytes
	ret.WriteBytes = now.WriteBytes - last.WriteBytes
	ret.BytesSent = now.BytesSent - last.BytesSent
	ret.BytesRecv = now.BytesRecv - last.BytesRecv
	ret.PacketsSent = now.PacketsSent - last.PacketsSent
	ret.PacketsRecv = now.PacketsRecv - last.PacketsRecv
	return ret
}

var FormatError = fmt.Errorf("Parse format error")

func ParseStatus(s []byte) (*Status, error) {
	ret := &Status{}
	lines := strings.Split(string(s), " ")
	if len(lines) < 42 {
		return nil, FormatError
	}
	if c, err := strconv.Atoi(lines[19]); err != nil {
		return nil, FormatError
	} else {
		ret.ThreadCount = int32(c)
	}
	var i int64
	var err error
	if i, err = strconv.ParseInt(lines[13], 0, 64); err != nil {
		return nil, FormatError
	}
	ret.UTime = i

	if i, err = strconv.ParseInt(lines[14], 0, 64); err != nil {
		return nil, FormatError
	}
	ret.STime = i

	if i, err = strconv.ParseInt(lines[15], 0, 64); err != nil {
		return nil, FormatError
	}
	ret.CuTime = i

	if i, err = strconv.ParseInt(lines[16], 0, 64); err != nil {
		return nil, FormatError
	}
	ret.CsTime = i
	return ret, nil
}

var last map[int]*Status = make(map[int]*Status)

func GetStatus(pid int, port string) (*Status, error) { //user state ctime
	if pid <= 1 {
		return &Status{}, fmt.Errorf("pid is 0 or 1")
	}
	//cpu
	b, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
	if err != nil {
		return nil, err
	}
	s, err := ParseStatus(b)
	s.ProcCount = 1
	if err != nil {
		return nil, err
	}
	cmdstr := `awk '{if ($1 == "cpu") {sum = $2 + $3 + $4 + $5 + $6 + $7 + $8 + $9 + $10 + $11;print sum}}' /proc/stat`
	cmdx := exec.Command(`/bin/sh`, `-c`, cmdstr)
	outx, _ := cmdx.CombinedOutput()
	cmdoutx := string(outx)
	CpuTime, _ := strconv.ParseInt(strings.TrimSpace(cmdoutx), 0, 64)
	s.CheckTime = CpuTime

	p, err := process.NewProcess(int32(pid))
	if err != nil {
		return nil, err
	}
	//memory
	ms, err := p.MemoryInfo()
	s.MemSize = int64(ms.RSS)

	//netIO
	if port == "" {
		s.BytesRecv = 0
		s.BytesSent = 0
		s.PacketsRecv = 0
		s.PacketsSent = 0
	} else {
		ports := strings.Split(port, `,`)
		for _, p := range ports {
			cmd := exec.Command(`iptables`, `-C`, `OUTPUT`, `-p`, `tcp`, `--sport`, p)
			err = cmd.Run()
			if err != nil {
				cmd = exec.Command(`iptables`, `-I`, `OUTPUT`, `-p`, `tcp`, `--sport`, p)
				cmd.Run()
			}

			cmd = exec.Command(`iptables`, `-C`, `INPUT`, `-p`, `tcp`, `--dport`, p)
			err = cmd.Run()
			if err != nil {
				cmd = exec.Command(`iptables`, `-I`, `INPUT`, `-p`, `tcp`, `--dport`, p)
				cmd.Run()
			}

			var out bytes.Buffer
			cmd = exec.Command(`/bin/sh`, `-c`, `iptables -L OUTPUT -n -v -x|grep `+p+`|awk 'NR==1{print $1","$2}'`)
			cmd.Stdout = &out
			err = cmd.Run()
			checkErr(err, "run proc cmd[iptables] error!")
			sent := out.String()
			out.Reset()
			ss := strings.Split(strings.TrimSpace(sent), `,`)
			if len(ss) != 2 {
				log.Errorf("port:%s has no data!", p)
				s.PacketsSent = 0
				s.BytesSent = 0
			} else {
				sPackets, _ := strconv.ParseUint(ss[0], 10, 64)
				s.PacketsSent = s.PacketsSent + sPackets
				sBytes, _ := strconv.ParseUint(strings.TrimSpace(ss[1]), 10, 64)
				s.BytesSent = s.BytesSent + sBytes
			}

			cmd = exec.Command(`/bin/sh`, `-c`, `iptables -L INPUT -n -v -x|grep `+p+`|awk 'NR==1{print $1","$2}'`)
			cmd.Stdout = &out
			err = cmd.Run()
			checkErr(err, "run proc cmd[iptables] error!")
			recv := out.String()
			out.Reset()
			rs := strings.Split(recv, `,`)
			if len(rs) != 2 {
				log.Errorf("port:%s has no data!", p)
				s.PacketsRecv = 0
				s.BytesRecv = 0
			} else {
				rPackets, _ := strconv.ParseUint(rs[0], 10, 64)
				s.PacketsRecv = s.PacketsRecv + rPackets
				rBytes, _ := strconv.ParseUint(strings.TrimSpace(rs[1]), 10, 64)
				s.BytesRecv = s.BytesRecv + rBytes
			}
		}
	}

	//diskIO
	dcs, err := p.IOCounters()
	if err != nil {
		return nil, err
	}
	s.ReadCount = dcs.ReadCount
	s.WriteCount = dcs.WriteCount
	s.ReadBytes = dcs.ReadBytes
	s.WriteBytes = dcs.WriteBytes

	if _, ok := last[pid]; !ok {
		last[pid] = &Status{}
	}
	ret := ProcCounterMinus(last[pid], s)
	last[pid] = s

	cmdstr3 := `grep -c processor /proc/cpuinfo`
	cmd3 := exec.Command(`/bin/sh`, `-c`, cmdstr3)
	out3, _ := cmd3.CombinedOutput()
	cmdout3 := string(out3)
	cores, _ := strconv.ParseFloat(strings.TrimSpace(cmdout3), 64)

	if ret.CheckTime == 0 {
		ret.CpuUsage = 0
	} else {
		ret.CpuUsage = float64(ret.CuTime+ret.CsTime+ret.UTime+ret.STime) / float64(ret.CheckTime) * cores
	}
	return ret, nil
}

func ParseCmdline(pid int) (content []byte, ctime int64, err error) {
	fn := fmt.Sprintf("/proc/%d/cmdline", pid)
	content, err = ioutil.ReadFile(fn)
	if err != nil {
		return
	}
	var fi os.FileInfo
	fi, err = os.Stat(fn)
	if err != nil {
		return
	}
	stat := fi.Sys().(*syscall.Stat_t)
	ctime = int64(stat.Ctim.Sec)
	return
}

func parse(stat string) (speed float64, err error) {
	str := strings.Split(stat, `:`)
	s := strings.TrimSpace(str[1])
	ss := strings.Trim(s, `Mb/s`)
	speed, err = strconv.ParseFloat(ss, 64)
	return
}

func CounterMinus(last, now map[string]uint64) map[string]float64 {
	ret := make(map[string]float64)
	if last == nil {
		return ret
	}
	for k, v := range now {
		lastV, ok := last[k]
		if ok {
			ret[k] = float64(v - lastV)
		}
	}
	return ret
}

func PercentMinus(last, now map[string]float64) map[string]float64 {
	lt, ok1 := last["total"]
	nt, ok2 := now["total"]
	ret := make(map[string]float64)
	if ok1 && ok2 {
		total := float64(nt - lt)
		for k, v := range now {
			if k == "total" {
				continue
			}
			if lastv, ok := last[k]; ok {
				ret[k] = float64(v-lastv) / total
			}
		}
	}
	ret["used"] = 1 - ret["idle"]
	return ret
}

func (pi *ProcessItem) Reset() {
	if pi.MatchMulti {
		pi.process = &MultiProcess{}
	} else {
		pi.process = &SingleProcess{}
	}
}

func (sp *SingleProcess) Uptime() int64 {
	return time.Now().Unix() - sp.ctime
}

func (sp *SingleProcess) Status(port string) (*Status, error) {
	if sp.pid == 0 {
		return nil, nil
	}
	stat, err := GetStatus(sp.pid, port)
	checkErr(err, "GetStatus error!")
	sp.checkTime = time.Now().Unix()
	stat.ProcType = "Single"
	return stat, err
}

func (sp *SingleProcess) Add(ctime int64, pid int) (err error) {
	if sp.pid == 0 {
		sp.pid = pid
		sp.ctime = ctime
		sp.checkTime = time.Now().Unix()
		return
	}
	return fmt.Errorf("Duplicate Process Matched In Single define")
}

func (sp *MultiProcess) Uptime() int64 {
	var sum int64
	var count int
	for _, v := range sp.processes {
		count += 1
		sum += time.Now().Unix() - v.ctime
	}
	return sum / int64(count)
}

func (sp *MultiProcess) Add(ctime int64, pid int) error {
	p := &SingleProcess{}
	if err := p.Add(ctime, pid); err != nil {
		return err
	}
	sp.processes = append(sp.processes, p)
	return nil
}

func Addto(t *Status, o *Status) {
	if o == nil {
		return
	}
	t.CuTime = t.CuTime + o.CuTime
	t.CsTime = t.CsTime + o.CsTime
	t.STime = t.STime + o.STime
	t.UTime = t.UTime + o.UTime
	t.CpuUsage = t.CpuUsage + o.CpuUsage
	t.MemSize = t.MemSize + o.MemSize
	t.ThreadCount = t.ThreadCount + o.ThreadCount
	t.ProcCount = t.ProcCount + o.ProcCount
	t.ReadBytes = t.ReadBytes + o.ReadBytes
	t.WriteBytes = t.WriteBytes + o.WriteBytes
	t.ReadCount = t.ReadCount + o.ReadCount
	t.WriteCount = t.WriteCount + o.WriteCount
	t.BytesSent = t.BytesSent + o.BytesSent
	t.BytesRecv = t.BytesRecv + o.BytesRecv
	t.PacketsSent = t.PacketsSent + o.PacketsSent
	t.PacketsRecv = t.PacketsRecv + o.PacketsRecv
}

func (sp *MultiProcess) Status(port string) (*Status, error) {
	stat := &Status{}
	for _, p := range sp.processes {
		s, err := p.Status(port)
		if err != nil {
			return nil, err
		}
		if !IsNull(s) {
			Addto(stat, s)
		}
	}
	stat.ProcType = "Multi"
	return stat, nil
}
